# -*- coding: utf-8 -*-

import scrapy
import re
import datetime
from test1.items import txjs


class txjsspider(scrapy.Spider):
    name = 'txjs'

    def start_requests(self):
        yield scrapy.FormRequest('http://202.106.120.64/scp/generalquery/cominfo/listAllCert.jsp',
                                formdata={
                                    'pageNum':'1'
                              },
                                callback=self.parse)

    def parse(self, response):
        x = response.xpath("//td[@class = 'table2'][2]/span/text()").extract_first()
        #总页数
        page_num =re.findall(u"(\d+)",x)[2]
        page_num = int(page_num)
        for i in range (1,page_num+1):
            yield scrapy.FormRequest('http://202.106.120.64/scp/generalquery/cominfo/listAllCert.jsp',
                                    formdata={
                                        'pageNum':str(i)
                                  },
                                    callback=self.parse_info,dont_filter=True)

    def parse_info(self,response):
        trs = response.xpath("//tr[@bgcolor = '#FFFFFF']")
        for tr in trs:
            if tr.xpath("td[1]/span/text()").extract_first():
                cominfo = txjs()
                cominfo['qualification_type'] = tr.xpath("td[1]/span/text()").extract_first()
                cominfo['the_name_of_the_enterprise'] = tr.xpath("td[2]/span/text()").extract_first()
                cominfo['qualification_certificate_number'] = tr.xpath("td[3]/span/text()").extract_first()
                cominfo['the_level_of_qualification'] = tr.xpath("td[4]/span/text()").extract_first()
                cominfo['the_date_of_issuance'] = tr.xpath("td[5]/span/text()").extract_first()
                cominfo['the_certificate_is_valid'] = tr.xpath("td[6]/span/text()").extract_first()
                cominfo['scope_of_business'] = tr.xpath("td[7]/span/text()").extract_first()
                cominfo['url'] = response.url
                cominfo['insert_time'] = str(datetime.datetime.now())
                yield cominfo
